# -*- coding: utf-8 -*-
"""
Spyder Editor


This is a temporary script file.
"""
import sys
sys.path.append("D:\Project\Argo")
sys.path.append("D:\Program Files\Python37\Lib\site-packages")
import datetime
import os
import pandas as pd
from sklearn.utils import shuffle
from sklearn import tree
#from sklearn.decomposition import PCA
from sklearn import preprocessing
import numpy as np
from sklearn.metrics import accuracy_score, auc, confusion_matrix, f1_score, precision_score, recall_score,roc_curve  # 导入指标库
import prettytable  # 导入表格库
import pydotplus  # 导入dot插件库
import matplotlib.pyplot as plt # 导入图形展示库
import copy
import pymongo
from sklearn.preprocessing import label_binarize
from sklearn.cross_validation import cross_val_score
import Core.Gadget as Gadget
#时间序列数据


# 获取train_y, train_x, test_y, test_x
def GetTrainData(dt):  #dt是当月，即test_data的月份

    # 取过去半年数据作为train_data,一个月数据作为test_data
    data_test = pd.read_csv('D:/StrategyData' + '/' + 'strategy_data_' + str(dt)[0:4] + str(dt)[5:7] + '.csv', encoding = 'GBK').set_index('Unnamed: 0')
    data_test['date'] = str(dt)[0:4] + str(dt)[5:7]

    referenceDate = dt + datetime.timedelta(days=-210)
    recentMonths = Gadget.GenerateEndDayofMonth(referenceDate, dt)[0:6]
    data_train = pd.DataFrame()
    for time in recentMonths:
        train = pd.read_csv('D:/StrategyData' + '/' + 'strategy_data_' + str(time)[0:4] + str(time)[5:7] + '.csv', encoding = 'GBK').set_index('Unnamed: 0')
        train['date'] = str(time)[0:4] + str(time)[5:7]
        data_train = pd.concat([data_train, train], axis = 0)

    # 将行业文字改为数字标记
    industry_list = [u'采掘', u'传媒', u'电气设备', u'电子', u'房地产', u'纺织服装', u'非银金融',
                     u'钢铁', u'公共事业', u'国防军工', u'化工', u'机械设备', u'计算机', u'家用电器',
                     u'建筑材料', u'建筑建材', u'建筑装饰', u'交通运输', u'交运设备', u'金融服务', u'农林牧渔',
                     u'汽车', u'轻工制造', u'商业贸易', u'食品饮料', u'通信', u'信息服务', u'信息设备',
                     u'休闲服务', u'医药生物', u'银行', u'有色金属', u'综合', u'公用事业']

    data_train = data_train.dropna()
    data_test = data_test.dropna()

    for i in range(len(industry_list)):
        insdustry = industry_list[i]
        data_train['industry'] = data_train['industry'].replace(insdustry, i)
        data_test['industry'] = data_test['industry'].replace(insdustry, i)

    # 这些因子数据存在inf，需剔除以免影响计算
    #train_data = train_data.reindex(range(len(train_data)))
    xlist = ['pricesales', 'GrossProfitMarginTTM', 'ReceivablesTurnoverTTM', 'ROETTM']
    for x in xlist:
        data_train = data_train[data_train[x] != np.inf]
        data_train = data_train[data_train[x] != -np.inf]
        data_test = data_test[data_test[x] != np.inf]
        data_test = data_test[data_test[x] != -np.inf]

    # 因子需取倒数
    backlist = ['pricecashflow', 'earningnetincome', 'operatingprofit', 'freecashflow']
    for back in backlist:
        data_train[back] = 1.0 / data_train[back]
        data_test[back] = 1.0 / data_test[back]

    # 将train_data和test_data一起做归一化
    data_train['lab'] = 0       #拼起来归一化，为了之后分开
    data_test['lab'] = 1

    data_all = pd.concat([data_train, data_test], axis=0)
    data_all_copy = copy.deepcopy(data_all)
    data_all = data_all.drop(columns = ['return', 'MonthlyExcessReturn', 'industry', 'lab', 'date'])
    data_all = (data_all - data_all.mean()) / data_all.std()
    data_all['industry'] = data_all_copy['industry']
    data_all['lab'] = data_all_copy['lab']
    data_all['return'] = data_all_copy['return']
    data_all['MonthlyExcessReturn'] = data_all_copy['MonthlyExcessReturn']
    data_all['date'] = data_all_copy['date']

    data_train0 = data_all[data_all['lab'] == 0]
    data_test0 = data_all[data_all['lab'] == 1]

    data_train_copy = copy.deepcopy(data_train0)
    data_test_copy = copy.deepcopy(data_test0)

    # 月度超额收益领涨20%的标记为1，领跌20%的标记为-1，其余标记0
    lim1 = np.percentile(np.array(data_train_copy[['MonthlyExcessReturn']]), 20)
    lim2 = np.percentile(np.array(data_train_copy[['MonthlyExcessReturn']]), 80)

    data_train0['return_tab'] = np.nan
    data_train0['return_tab'][data_train0['MonthlyExcessReturn'] >= lim2] = 1
    data_train0['return_tab'][(data_train0['MonthlyExcessReturn'] > lim1) & (data_train0['MonthlyExcessReturn'] < lim2)] = 0
    data_train0['return_tab'][data_train0['MonthlyExcessReturn'] <= lim1] = -1

    data_test0['return_tab'] = np.nan
    data_test0['return_tab'][data_test0['MonthlyExcessReturn'] >= lim2] = 1
    data_test0['return_tab'][(data_test0['MonthlyExcessReturn'] > lim1) & (data_test0['MonthlyExcessReturn'] < lim2)] = 0
    data_test0['return_tab'][data_test0['MonthlyExcessReturn'] <= lim1] = -1

    data_train0 = data_train0.drop(columns = ['return', 'MonthlyExcessReturn', 'lab', 'date'])
    data_test0 = data_test0.drop(columns = ['return', 'MonthlyExcessReturn', 'lab', 'date'])

    train_y = data_train0[['return_tab']]
    train_x = data_train0.drop(columns = ['return_tab'])

    test_y = data_test0[['return_tab']]
    test_x = data_test0.drop(columns = ['return_tab'])

    return train_y, train_x, test_y, test_x, data_test_copy      #保留data_test_copy为了后续统计预测为1的股票的收益率


# 决策树最优层数调优
def BestDepthOfTreeModel(deplist, train_y, train_x, test_y, test_x):
    max_dep = []
    f1_list = []
    train_score = []
    test_score = []

    for dep in deplist:
        print('dep', dep)
        model = tree.DecisionTreeClassifier(max_depth=dep,min_samples_leaf=1,random_state=0)
        model.fit(train_x, train_y)
        predicted = model.predict(test_x)
        predict = pd.DataFrame(data = predicted, index = test_x.index, columns = ['predict'])

        train_s = model.score(train_x, train_y)
        test_s = accuracy_score(test_y, predict)
        train_score.append([dep,train_s])
        test_score.append([dep,test_s])

        f1_s = f1_score(test_y, predict, average='weighted')
        f1_list.append(f1_s)

    best_dep = deplist[f1_list.index(max(f1_list))]    #这里用f1得分来选，也可以自定义取最优层数的算法
    print('f1_score', best_dep)

    a = pd.DataFrame(train_score)
    a.columns = ['model_complexity', 'train_score']
    a = a.set_index('model_complexity')
    b = pd.DataFrame(test_score)
    b.columns = ['model_complexity', 'test_score']
    b = b.set_index('model_complexity')
    c = pd.concat([a,b],axis=1)
    c.plot()

    return best_dep, c


# 得到最优层数开始训练模型
def TreeModel(best_dep, train_y, train_x, test_y, test_x, dt, data_test_copy):   #best_dep = 5
    info = []
    Model = tree.DecisionTreeClassifier(max_depth=best_dep,min_samples_leaf=1,random_state=0)
    Model.fit(train_x, train_y)
    Predicted = Model.predict(test_x)
    Predict = pd.DataFrame(data = Predicted, index = test_x.index, columns = ['predict'])
    selstock = Predict[Predict['predict'] == 1]
    #print (selstock)
    stocklist = selstock.index.tolist()    #选出预测为1的股票

    test_s = accuracy_score(test_y, Predict)   #准确率
    precision_s = precision_score(test_y, Predict, average='weighted')  # 精确度
    recall_s = recall_score(test_y, Predict, average='weighted')   #召回率
    f1_s = f1_score(test_y, Predict, average='weighted')    #f1得分

    dot_data = tree.export_graphviz(Model, out_file=None, feature_names=train_x.columns.tolist(), max_depth=best_dep, filled=True,rounded=True, class_names = ['-1', '0', '1'])  # 将决策树规则生成dot对象
    graph = pydotplus.graph_from_dot_data(dot_data)  # 通过pydotplus将决策树规则解析为图形
    graph.write_pdf('D:/model_result/trees5/' + str(dt)[0:4] + str(dt)[5:7] + '.pdf')  # 将决策树规则保存为PDF文件

    if len(stocklist) != 0:
        select = data_test_copy.ix[stocklist, ['return', 'MonthlyExcessReturn']]
        info.append([str(dt)[0:7], select.mean()[0], select.mean()[1], len(select), best_dep, test_s, precision_s, recall_s,f1_s])
    else:
        info.append([str(dt)[0:7], 0, 0, 0, best_dep, test_s, precision_s, recall_s, f1_s])

    return stocklist, info



def BestDepthOfTreeModel_Batch(datetimes, deplist):
    for dt in datetimes:
        train_y, train_x, test_y, test_x, data_test_copy = GetTrainData(dt)
        best_dep, c = BestDepthOfTreeModel(deplist, train_y, train_x, test_y, test_x)
        print(dt, best_dep)


def TreeModel_Batch(datetimes, best_dep):
    info_list = []
    for dt in datetimes:
        train_y, train_x, test_y, test_x, data_test_copy = GetTrainData(dt)
        stocklist, info = TreeModel(best_dep, train_y, train_x, test_y, test_x, dt, data_test_copy)

        symbols_table = pd.DataFrame(stocklist)
        symbols_table.to_csv('D:/model_result/tree_position5/' + str(dt)[0:4] + str(dt)[5:7] + '.csv',
                             encoding='GBK')  # 将每个月持仓存起来
        info_list.append(info)

    infotable = pd.DataFrame(info,
                             columns=['time', 'return', 'excessreturn', 'num', 'best_dep', 'test_score', 'precision',
                                      'recall', 'f1'])
    return_info = infotable[['return', 'excessreturn']]
    cumprod_return = pd.DataFrame.cumprod(1 + return_info)
    cumprod_return.plot()

    return infotable


datetime1 = datetime.datetime(2010, 7, 30)
datetime2 = datetime.datetime(2018, 9, 1)
datetime1 = Gadget.ToUTCDateTime(datetime1)
datetime2 = Gadget.ToUTCDateTime(datetime2)
datetimes = Gadget.GenerateEndDayofMonth(datetime1, datetime2)
deplist = [3, 5, 8, 10, 15]

# 1. 决策树最优层数选择
BestDepthOfTreeModel_Batch(datetimes, deplist)


# 2. 决策树模型训练
TreeModel_Batch(datetimes, best_dep=5)






# 以下是草稿
'''
# 混淆矩阵
confusion_m = confusion_matrix(test_data[['return_tab']], predict)  # 获得混淆矩阵
confusion_matrix_table = prettytable.PrettyTable()  # 创建表格实例
confusion_matrix_table.add_row(confusion_m[0, :])  # 增加第一行数据
confusion_matrix_table.add_row(confusion_m[1, :])  # 增加第二行数据
confusion_matrix_table.add_row(confusion_m[2, :])  # 增加第3行数据
#confusion_matrix_table.add_row(confusion_m[3, :])  # 增加第4行数据
print ('confusion matrix')
print (confusion_matrix_table)  # 打印输出混淆矩阵

#分行业
data_train = pd.DataFrame()
data_test = pd.DataFrame()
for industry in industry_list:
    data_industry = data_table[data_table['industry'] == industry]#.drop(columns = ['Unnamed: 0', 'industry'])
    data_shuffle = shuffle(data_industry)
    train_num = int(len(data_shuffle.index)*0.7)
    test_num = len(data_shuffle.index) - train_num
    train_sel = data_shuffle.iloc[0:train_num, :]
    test_sel = data_shuffle.iloc[-test_num:, :]
    data_train = pd.concat([data_train, train_sel], axis = 0)#.reset_index()
    data_test = pd.concat([data_test, test_sel], axis = 0)#.reset_index()
    

#分行业训练
Train_data = copy.deepcopy(train_data)
Test_data = copy.deepcopy(test_data)
Train_data = train_data - train_data.mean() / train_data.std()
Train_data['earning'] = train_data['earning']
Train_data['industry'] = train_data['industry']
Train_data['return_tab'] = train_data['return_tab']
Test_data = test_data - test_data.mean() / test_data.std()
Test_data['earning'] = test_data['earning']
Test_data['industry'] = test_data['industry']
Test_data['return_tab'] = test_data['return_tab']
Tree_table = pd.DataFrame()
for ind in range(33):
#for ind in range(7,8):
    print (ind)
    Train_sel = Train_data[Train_data['industry'] == ind]
    Train_x = Train_sel.drop(columns = ['return_tab'])
    Train_y = Train_sel[['return_tab']]
    Test_sel = Test_data[Test_data['industry'] == ind]
    Test_x = Test_sel.drop(columns = ['return_tab'])
    Model = tree.DecisionTreeClassifier(max_depth=5,min_samples_leaf=1,random_state=0)
    try:
        Model.fit(Train_x, Train_y)
    except:
        continue
    Predicted= Model.predict(Test_x)
    Predict = pd.DataFrame(data = Predicted, index = Test_sel.index, columns = ['Predict'])
    Tree_model = pd.concat([Predict, Test_sel[['return_tab']]], axis = 1)
    Tree_table = pd.concat([Tree_table, Tree_model], axis = 0)

precise = 1.0 - (Tree_table['Predict'] != Tree_table['return_tab']).sum() / len(Tree_table.index)


# 核心评估指标
y_score = model.predict_proba(test_x)  # 获得决策树的预测概率
#fpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])  # ROC
#auc_s = auc(fpr, tpr)  # AUC
accuracy_s = accuracy_score(test_data[['return_tab']], predict)  # 准确率
precision_s = precision_score(test_data[['return_tab']], predict)  # 精确度
recall_s = recall_score(test_data[['return_tab']], predict)  # 召回率
f1_s = f1_score(test_data[['return_tab']], predict)  # F1得分
core_metrics = prettytable.PrettyTable()  # 创建表格实例
core_metrics.field_names = ['auc', 'accuracy', 'precision', 'recall', 'f1']  # 定义表格列名
core_metrics.add_row([auc_s, accuracy_s, precision_s, recall_s, f1_s])  # 增加数据
print ('core metrics')
print (core_metrics)  # 打印输出核心评估指标
'''